# import tiktoken_ext.openai_public
# import inspect
#
# print(dir(tiktoken_ext.openai_public))
# # The encoder we want is cl100k_base, we see this as a possible function
#
# print(inspect.getsource(tiktoken_ext.openai_public.cl100k_base))

# import hashlib
#
# blobpath = "I:\\models\\damo"  # 这个例子里是/root/models
# cache_key = hashlib.sha1(blobpath.encode()).hexdigest()
# print(cache_key)

import os
import tiktoken

tiktoken_cache_dir = "I:\\models\\damo"
os.environ["TIKTOKEN_CACHE_DIR"] = tiktoken_cache_dir

# validate
# assert os.path.exists(os.path.join(tiktoken_cache_dir, cache_key))

encoding = tiktoken.get_encoding("cl100k_base")
ll = encoding.encode("Hello, world")
print(ll)
print(encoding.decode(ll))